package com.feng.mainPack;

import java.io.IOException;
import java.util.Iterator;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class GetPageUrl extends Thread {

	private String sUrl;

	public GetPageUrl(String sUrl) {
		this.sUrl = sUrl;
	}

	@Override
	public void run() {

		// 先执行一次，让下面的循环可以执行
		get(sUrl);

		while (QueueList.PageUrls.size() > 0) {

			String u = QueueList.PageUrls.poll();

			// 肯定是这个线程先执行，所以一旦链接被访问，说明肯定已经提取过其中下一页的链接
			if (QueueList.VisitiedPageUrls.contains(u) ||QueueList.PageUrls.contains(u)) {
				continue;
			}

			get(u);

			System.out.println("剩余pageUrl" + QueueList.PageUrls.size());

		}

	}

	public void get(String url) {

		System.out.println("正在从" + url + "获取页面链接");

		Document doc = null;
		try {
			doc = Jsoup.connect(url).timeout(30000).get();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

		if (doc.getElementById("nav_next_page") != null) { // 首页时执行这个

			Element e = doc.getElementById("nav_next_page");

			Iterator<Element> ie2 = e.getAllElements().iterator();

			while (ie2.hasNext()) {
				Element e1 = ie2.next();

				if (e1.tagName() == "a") {

					// System.out.print(e1.text()+":");
					// System.out.println(e1.attr("href"));
					String tu = e1.attr("href");

					if (tu == null || QueueList.VisitiedPageUrls.contains(tu)) {
						// 结束本次循环
						continue;
					}else{
						QueueList.PageUrls.offer(tu);
						
					}


				}

			}

		} else {

			Elements es = doc.getElementsByClass("pager");

			// System.out.println(ie.);

			if (es.size() != 0) { // 查看是否为第二页

				Iterator<Element> ie = es.iterator();

				while (ie.hasNext()) {
					Iterator<Element> ie2 = ie.next().getAllElements().iterator();

					while (ie2.hasNext()) {
						Element e1 = ie2.next();

						if (e1.tagName() == "a") {

							// System.out.print(e1.text()+":");
							// System.out.println(e1.attr("href"));

							String tu = e1.attr("href");

							if (QueueList.VisitiedPageUrls.contains(tu)) {
								// 结束本次循环
								continue;
							}else{
								
								QueueList.PageUrls.offer(tu);
							}


						}

					}
					// 程序仅循环一次
					// break;

				}
			}
		}
		System.out.println("获取成功");
	}

}
